{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 热身：使用NumPy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在介绍PyTorch之前，我们将首先使用NumPy实现网络。\n",
    "\n",
    "NumPy提供了一个n维数组对象和许多用于操作这些数组的函数。NumPy是用于科学计算的通用框架；它对计算图、深度学习和梯度一无所知。然而，我们可以很容易地使用NumPy，手动实现网络的前向和反向传播，来拟合随机数据："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 30809436.38152115\n",
      "1 23189235.61111492\n",
      "2 18954327.919291187\n",
      "3 15486515.426800951\n",
      "4 12115287.599843461\n",
      "5 8990009.98012044\n",
      "6 6385418.2103486005\n",
      "7 4437627.164542308\n",
      "8 3077915.682863354\n",
      "9 2171974.731766707\n",
      "10 1575843.7842849912\n",
      "11 1182734.6665274957\n",
      "12 917575.5934632119\n",
      "13 733601.5979606146\n",
      "14 601584.9442991028\n",
      "15 503355.00837866194\n",
      "16 427816.9305089206\n",
      "17 368040.34677498986\n",
      "18 319623.9642881076\n",
      "19 279705.9317797349\n",
      "20 246284.87453937682\n",
      "21 217955.47978031356\n",
      "22 193723.44316899264\n",
      "23 172833.3093086189\n",
      "24 154715.94822603732\n",
      "25 138936.00259415322\n",
      "26 125116.31386497474\n",
      "27 112960.89067049169\n",
      "28 102227.3773527094\n",
      "29 92713.37351109677\n",
      "30 84257.35485552298\n",
      "31 76708.55199371943\n",
      "32 69958.33147692484\n",
      "33 63905.75896586577\n",
      "34 58469.31478683617\n",
      "35 53575.41817596209\n",
      "36 49159.269202173295\n",
      "37 45167.73414989073\n",
      "38 41556.147806201814\n",
      "39 38277.84312810578\n",
      "40 35299.39839600997\n",
      "41 32588.40614867875\n",
      "42 30118.238388661768\n",
      "43 27863.539835716318\n",
      "44 25802.459674301877\n",
      "45 23916.343670869843\n",
      "46 22187.901462551883\n",
      "47 20601.55570199108\n",
      "48 19144.717993209906\n",
      "49 17805.671598708985\n",
      "50 16574.0487688064\n",
      "51 15438.594412157745\n",
      "52 14390.800224415814\n",
      "53 13423.70866564057\n",
      "54 12529.780529870048\n",
      "55 11703.272059604547\n",
      "56 10938.002116254982\n",
      "57 10229.041720672772\n",
      "58 9571.85742187299\n",
      "59 8961.733685399027\n",
      "60 8395.494840920426\n",
      "61 7868.988643850864\n",
      "62 7381.0270179330055\n",
      "63 6927.536829271761\n",
      "64 6505.200224685315\n",
      "65 6111.830768641336\n",
      "66 5745.004070256817\n",
      "67 5402.770683459274\n",
      "68 5083.288624089411\n",
      "69 4784.868818806751\n",
      "70 4506.028376447461\n",
      "71 4245.2123323531405\n",
      "72 4001.1212706758815\n",
      "73 3772.686834125283\n",
      "74 3558.6887591931154\n",
      "75 3358.100325965559\n",
      "76 3169.9962740389274\n",
      "77 2993.48630290624\n",
      "78 2827.8323887325296\n",
      "79 2672.2550902597\n",
      "80 2526.1611861713986\n",
      "81 2388.9967699647896\n",
      "82 2260.0882337936387\n",
      "83 2138.810558901033\n",
      "84 2024.6720237826012\n",
      "85 1917.2285218363043\n",
      "86 1816.0586179330312\n",
      "87 1720.6705141838597\n",
      "88 1630.736419603229\n",
      "89 1545.9537920209125\n",
      "90 1465.9776910397359\n",
      "91 1390.5142706081342\n",
      "92 1319.3002461227059\n",
      "93 1252.0726085487186\n",
      "94 1188.6062848837523\n",
      "95 1128.6420843011435\n",
      "96 1071.9659900794945\n",
      "97 1018.3729373958274\n",
      "98 967.6531518092656\n",
      "99 919.6655810227867\n",
      "100 874.2533734860046\n",
      "101 831.2768366367518\n",
      "102 790.5649277086289\n",
      "103 751.9808884545153\n",
      "104 715.4249487926754\n",
      "105 680.7727307553869\n",
      "106 647.9443365560596\n",
      "107 616.8042191522486\n",
      "108 587.2618951946193\n",
      "109 559.2383213030871\n",
      "110 532.6393782545165\n",
      "111 507.3893437271771\n",
      "112 483.4189287677314\n",
      "113 460.64964153011806\n",
      "114 439.0252362159699\n",
      "115 418.4788108828126\n",
      "116 398.9601032756123\n",
      "117 380.4086652417594\n",
      "118 362.76618063857825\n",
      "119 345.988787341531\n",
      "120 330.0322296474393\n",
      "121 314.85662033221814\n",
      "122 300.4195609968048\n",
      "123 286.68446462241087\n",
      "124 273.6130382845537\n",
      "125 261.163533474614\n",
      "126 249.3114048957239\n",
      "127 238.02497025001125\n",
      "128 227.28101507080208\n",
      "129 217.04299170367383\n",
      "130 207.2882238847346\n",
      "131 197.99420876132143\n",
      "132 189.1385918917686\n",
      "133 180.69807363219465\n",
      "134 172.6512044594672\n",
      "135 164.97859667226487\n",
      "136 157.66037900834638\n",
      "137 150.68372135491794\n",
      "138 144.02791772051205\n",
      "139 137.6804387129006\n",
      "140 131.62357458689647\n",
      "141 125.84563281266502\n",
      "142 120.3303854649473\n",
      "143 115.06528108744666\n",
      "144 110.04147563305361\n",
      "145 105.24640372323695\n",
      "146 100.66654064961192\n",
      "147 96.29277861688175\n",
      "148 92.11654025232994\n",
      "149 88.12826438517541\n",
      "150 84.32185908847508\n",
      "151 80.68305727103625\n",
      "152 77.20618987736519\n",
      "153 73.88463883558465\n",
      "154 70.71027000400167\n",
      "155 67.67698391194472\n",
      "156 64.77905026834733\n",
      "157 62.00814260865391\n",
      "158 59.35930189714148\n",
      "159 56.827633519241786\n",
      "160 54.40696904324366\n",
      "161 52.092728414594724\n",
      "162 49.879957241034546\n",
      "163 47.763453587315205\n",
      "164 45.73893144362265\n",
      "165 43.802598047811934\n",
      "166 41.95061057464524\n",
      "167 40.179770262704274\n",
      "168 38.48503252307472\n",
      "169 36.86421395418573\n",
      "170 35.31266394831976\n",
      "171 33.828097777959876\n",
      "172 32.40762867218753\n",
      "173 31.04845008473209\n",
      "174 29.747326239363193\n",
      "175 28.501933341640544\n",
      "176 27.309941058691976\n",
      "177 26.168812732671842\n",
      "178 25.076897762218415\n",
      "179 24.031456012889898\n",
      "180 23.045149669495366\n",
      "181 22.102340942725917\n",
      "182 21.199400471945495\n",
      "183 20.334669829481086\n",
      "184 19.50636448255984\n",
      "185 18.712629792067254\n",
      "186 17.95211213149509\n",
      "187 17.223233009596182\n",
      "188 16.524840209464937\n",
      "189 15.85561306446213\n",
      "190 15.214116440898561\n",
      "191 14.599013625217452\n",
      "192 14.009327142420368\n",
      "193 13.444005571099414\n",
      "194 12.902061519698096\n",
      "195 12.382504586515632\n",
      "196 11.884145722035957\n",
      "197 11.406247181989453\n",
      "198 10.947932399507422\n",
      "199 10.50838876650733\n",
      "200 10.087064298366613\n",
      "201 9.68271843944665\n",
      "202 9.294873188927843\n",
      "203 8.922824930449629\n",
      "204 8.565970588543728\n",
      "205 8.223601323308568\n",
      "206 7.895134759741055\n",
      "207 7.579957487612298\n",
      "208 7.277611977600579\n",
      "209 6.987437366505429\n",
      "210 6.709038002520496\n",
      "211 6.442069260682853\n",
      "212 6.185740076764391\n",
      "213 5.939711152865971\n",
      "214 5.7035909878608395\n",
      "215 5.477009176672871\n",
      "216 5.2595993619671635\n",
      "217 5.050872509011964\n",
      "218 4.850543724006152\n",
      "219 4.65827627681994\n",
      "220 4.473709079593329\n",
      "221 4.296565872273179\n",
      "222 4.126583605542429\n",
      "223 3.9633570020904614\n",
      "224 3.8066362131117675\n",
      "225 3.656179734215521\n",
      "226 3.5117740664679156\n",
      "227 3.3731249002393886\n",
      "228 3.239992336755944\n",
      "229 3.112162022729946\n",
      "230 2.9894270187056815\n",
      "231 2.8715848182614483\n",
      "232 2.758452214007096\n",
      "233 2.6498079986514993\n",
      "234 2.545486860324404\n",
      "235 2.4452983948473888\n",
      "236 2.349086799547189\n",
      "237 2.2567276842103796\n",
      "238 2.168001325378266\n",
      "239 2.0827833621176426\n",
      "240 2.000952198794305\n",
      "241 1.922360667067149\n",
      "242 1.8468977493381855\n",
      "243 1.7744057690114534\n",
      "244 1.7047844635545015\n",
      "245 1.6379405259639246\n",
      "246 1.5737054615902242\n",
      "247 1.5120330710833914\n",
      "248 1.4527755699656963\n",
      "249 1.3958558566490844\n",
      "250 1.3411826433272425\n",
      "251 1.2886703947192923\n",
      "252 1.2382399388673926\n",
      "253 1.189777331148488\n",
      "254 1.1432346624204608\n",
      "255 1.098519347984068\n",
      "256 1.055567864076434\n",
      "257 1.0143197695379689\n",
      "258 0.9746766967797451\n",
      "259 0.936593326366509\n",
      "260 0.9000035590642197\n",
      "261 0.8648584458579571\n",
      "262 0.8310941424974263\n",
      "263 0.7986560950909144\n",
      "264 0.7674857185132324\n",
      "265 0.7375393522491324\n",
      "266 0.7087698330787001\n",
      "267 0.6811298438022424\n",
      "268 0.6545775597188033\n",
      "269 0.6290596595300448\n",
      "270 0.6045405646343713\n",
      "271 0.5809864577169787\n",
      "272 0.5583542823695025\n",
      "273 0.5366035492646234\n",
      "274 0.5157048976252906\n",
      "275 0.49562633753860214\n",
      "276 0.47633317579128265\n",
      "277 0.45779218940337385\n",
      "278 0.43997733334669836\n",
      "279 0.42285727979736537\n",
      "280 0.4064122970133943\n",
      "281 0.3906085780783476\n",
      "282 0.37541843528512675\n",
      "283 0.3608197391119887\n",
      "284 0.3467923590891943\n",
      "285 0.3333215595852498\n",
      "286 0.3203679228661938\n",
      "287 0.30791874582081674\n",
      "288 0.29595574424336235\n",
      "289 0.28445947240499325\n",
      "290 0.27341285991162456\n",
      "291 0.2627971315518095\n",
      "292 0.2525944432147821\n",
      "293 0.24278725541234547\n",
      "294 0.2333645092719166\n",
      "295 0.22430719872181468\n",
      "296 0.2156024472700394\n",
      "297 0.20723637972106387\n",
      "298 0.19919698885259185\n",
      "299 0.19147075227498095\n",
      "300 0.1840445560212968\n",
      "301 0.1769071361675972\n",
      "302 0.17004684887734633\n",
      "303 0.16345611440824429\n",
      "304 0.1571198663182748\n",
      "305 0.15102908556851596\n",
      "306 0.145175331688285\n",
      "307 0.13955000431159914\n",
      "308 0.1341428121792559\n",
      "309 0.12894540999541493\n",
      "310 0.12395038907361305\n",
      "311 0.11914873079818614\n",
      "312 0.11453431721958769\n",
      "313 0.11009862057681151\n",
      "314 0.1058349702843717\n",
      "315 0.10173771928216985\n",
      "316 0.09779944222415599\n",
      "317 0.09401336750850982\n",
      "318 0.0903740839677914\n",
      "319 0.08687629361975269\n",
      "320 0.08351440813379896\n",
      "321 0.08028238518360487\n",
      "322 0.0771760405220862\n",
      "323 0.07418977658394174\n",
      "324 0.07131979824350618\n",
      "325 0.06856089178568926\n",
      "326 0.06590907528147491\n",
      "327 0.06336013121842274\n",
      "328 0.06091009144707986\n",
      "329 0.05855480640704243\n",
      "330 0.056290396281968226\n",
      "331 0.05411388230814554\n",
      "332 0.052021855473996584\n",
      "333 0.05001083620036004\n",
      "334 0.04807753144589756\n",
      "335 0.046219076418509863\n",
      "336 0.044432767093158555\n",
      "337 0.04271567277516629\n",
      "338 0.04106545260113244\n",
      "339 0.039478522811408436\n",
      "340 0.03795312653332559\n",
      "341 0.03648689162478724\n",
      "342 0.03507712579949075\n",
      "343 0.03372203412657043\n",
      "344 0.03241941145193486\n",
      "345 0.031167222209837402\n",
      "346 0.02996337312052379\n",
      "347 0.028806161502842497\n",
      "348 0.02769378012598517\n",
      "349 0.026624471994658522\n",
      "350 0.025596531678494194\n",
      "351 0.02460810616074408\n",
      "352 0.02365805982393626\n",
      "353 0.022744704954165352\n",
      "354 0.02186654598845591\n",
      "355 0.02102242916076775\n",
      "356 0.020210985372932967\n",
      "357 0.019430908934173215\n",
      "358 0.018680907453491\n",
      "359 0.017959911515345747\n",
      "360 0.01726685971466897\n",
      "361 0.016600638392088377\n",
      "362 0.015960023527037688\n",
      "363 0.015344127687047158\n",
      "364 0.01475213962050376\n",
      "365 0.01418294062571043\n",
      "366 0.013635684867821136\n",
      "367 0.013109677796698638\n",
      "368 0.012603972564239129\n",
      "369 0.012117743361866933\n",
      "370 0.011650281395546557\n",
      "371 0.011200938340758037\n",
      "372 0.010768953349078257\n",
      "373 0.010353640061409167\n",
      "374 0.009954308984707517\n",
      "375 0.00957042879055471\n",
      "376 0.009201359741440039\n",
      "377 0.008846516212035603\n",
      "378 0.008505391788522165\n",
      "379 0.008177459964805704\n",
      "380 0.007862127069369033\n",
      "381 0.007558966626774664\n",
      "382 0.0072675555092787366\n",
      "383 0.006987385966592398\n",
      "384 0.006718024669591033\n",
      "385 0.006459036923986605\n",
      "386 0.006210065660621858\n",
      "387 0.005970668877516466\n",
      "388 0.005740535117792721\n",
      "389 0.0055192496753159815\n",
      "390 0.0053065310747066\n",
      "391 0.005101992301472006\n",
      "392 0.0049053371625722856\n",
      "393 0.004716301061218366\n",
      "394 0.004534537917346419\n",
      "395 0.004359815433907501\n",
      "396 0.004191803971456638\n",
      "397 0.004030276054140406\n",
      "398 0.003874975685044752\n",
      "399 0.0037256578760917654\n",
      "400 0.003582112646657154\n",
      "401 0.0034440766838261033\n",
      "402 0.0033113818009325655\n",
      "403 0.003183794605894824\n",
      "404 0.00306113229686056\n",
      "405 0.0029431975541502607\n",
      "406 0.0028298231426726107\n",
      "407 0.002720821688973885\n",
      "408 0.0026160072248887865\n",
      "409 0.0025152283905961197\n",
      "410 0.002418341122426419\n",
      "411 0.0023251879471333017\n",
      "412 0.0022356163751285725\n",
      "413 0.0021495036187261533\n",
      "414 0.0020667168362667964\n",
      "415 0.0019871069441138563\n",
      "416 0.001910571687163259\n",
      "417 0.0018370044308316825\n",
      "418 0.0017662561528413413\n",
      "419 0.0016982373349997942\n",
      "420 0.0016328299550108284\n",
      "421 0.0015699514247957658\n",
      "422 0.0015094887628807523\n",
      "423 0.0014513560640996544\n",
      "424 0.0013954708773692608\n",
      "425 0.0013417312255285373\n",
      "426 0.0012900616546313386\n",
      "427 0.0012403892314361692\n",
      "428 0.0011926405249224387\n",
      "429 0.0011467165653710654\n",
      "430 0.001102565871224142\n",
      "431 0.0010601140908335271\n",
      "432 0.0010192934371243399\n",
      "433 0.0009800508244934205\n",
      "434 0.0009423175687962818\n",
      "435 0.0009060363407049964\n",
      "436 0.0008711543243246868\n",
      "437 0.0008376159529792463\n",
      "438 0.0008053692451289995\n",
      "439 0.0007743725080863267\n",
      "440 0.0007445632614751785\n",
      "441 0.0007158996980059986\n",
      "442 0.0006883396847422282\n",
      "443 0.0006618420379605795\n",
      "444 0.0006363655715310768\n",
      "445 0.0006118685354992003\n",
      "446 0.0005883150298965959\n",
      "447 0.0005656713371320302\n",
      "448 0.000543895564359084\n",
      "449 0.0005229609977567427\n",
      "450 0.0005028393180270419\n",
      "451 0.00048348300100678704\n",
      "452 0.00046487388217454373\n",
      "453 0.0004469825363442094\n",
      "454 0.00042977796447740886\n",
      "455 0.00041323663732546246\n",
      "456 0.000397332227641487\n",
      "457 0.00038204030016929767\n",
      "458 0.0003673361135937304\n",
      "459 0.00035319976680668716\n",
      "460 0.00033960792791787043\n",
      "461 0.0003265406221039206\n",
      "462 0.0003139741815307238\n",
      "463 0.000301891219667264\n",
      "464 0.00029027307345238156\n",
      "465 0.00027910254302514003\n",
      "466 0.0002683625371853718\n",
      "467 0.0002580345549389357\n",
      "468 0.00024810528610434387\n",
      "469 0.00023855838034594502\n",
      "470 0.00022937790080227797\n",
      "471 0.00022055344004900753\n",
      "472 0.00021206781538003736\n",
      "473 0.0002039069915561948\n",
      "474 0.00019606130710053343\n",
      "475 0.00018851700725031095\n",
      "476 0.00018126326274767656\n",
      "477 0.00017428861531787633\n",
      "478 0.00016758264818565437\n",
      "479 0.00016113443010223438\n",
      "480 0.00015493469357615597\n",
      "481 0.00014897378048415956\n",
      "482 0.0001432429981790996\n",
      "483 0.000137732170956133\n",
      "484 0.00013243274747242443\n",
      "485 0.00012733759689519055\n",
      "486 0.00012243827757282597\n",
      "487 0.00011772768364722118\n",
      "488 0.00011319833779597588\n",
      "489 0.00010884303453465466\n",
      "490 0.00010465585913112539\n",
      "491 0.00010062936198204087\n",
      "492 9.675841379307044e-05\n",
      "493 9.303690907640828e-05\n",
      "494 8.945737660119255e-05\n",
      "495 8.601601922242472e-05\n",
      "496 8.270696285973125e-05\n",
      "497 7.952517138233703e-05\n",
      "498 7.646586052323305e-05\n",
      "499 7.352423759871178e-05\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 two_layer_net_numpy.py\n",
    "import numpy as np\n",
    "\n",
    "# N是批大小；D_in是输入维度\n",
    "# H是隐藏层维度；D_out是输出维度  \n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生随机输入和输出数据\n",
    "x = np.random.randn(N, D_in)\n",
    "y = np.random.randn(N, D_out)\n",
    "\n",
    "# 随机初始化权重\n",
    "w1 = np.random.randn(D_in, H)\n",
    "w2 = np.random.randn(H, D_out)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # 前向传播：计算预测值y\n",
    "    h = x.dot(w1)\n",
    "    h_relu = np.maximum(h, 0)\n",
    "    y_pred = h_relu.dot(w2)\n",
    "\n",
    "    # 计算并显示loss（损失）\n",
    "    loss = np.square(y_pred - y).sum()\n",
    "    print(t, loss)\n",
    "\n",
    "    # 反向传播，计算w1、w2对loss的梯度\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.T.dot(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.dot(w2.T)\n",
    "    grad_h = grad_h_relu.copy()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.T.dot(grad_h)\n",
    "\n",
    "    # 更新权重\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (Spyder)",
   "language": "python3",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "227.797px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
